In [ ]:
# Purpose of the code:
# Play around with the dataset (FeatureDay)
# Generate some preliminary plots
# Plot scatter plots to track change of features over days of study
# plot and Fit regression lines to feature values over days of study
In [1]:
# necessary imports
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import math as math
from scipy.stats import pearsonr, betai
%matplotlib inline
In [2]:
# switch to the proper directory to access the data
pwd
Out[2]:
In [3]:
cd /camdatalake/bronze/verily_ms/device/
In [4]:
pwd
Out[4]:
In [5]:
# check the content of the directory
ls
In [6]:
# download and read the data
# FeatureDay: Average value of the features for each day of study. Days are listed as
# DayOfStudy
# FeatureStudy: Features for the entire study period.For the at home features,
# the reported value is the median of the observed day level values.
import gzip, csv
with gzip.open("FeaturesDay.csv.gz", "rt", newline="") as file:
FeatureDay = pd.read_csv(file, header = 0)
with gzip.open("FeaturesStudy.csv.gz", "rt", newline="") as file:
FeatureStudy = pd.read_csv(file, header = 0)
In [7]:
# explore the dataset
FeatureDay.info()
In [8]:
FeatureDay.describe()
Out[8]:
In [9]:
FeatureDay.head()
Out[9]:
In [10]:
# extract name of features
list(FeatureDay.columns)
Out[10]:
In [13]:
# found list of unique IDs for patients
patient_IDs = list(FeatureDay['gls_subject_code'].unique())
patient_IDs
Out[13]:
In [15]:
# 10 free living feature with high correlation
# ['idle_minutes',
# 'turn_vel_std_ankle',
# 'swing',
# 'stance',
# 'duration_movement_count',
# 'turn_vel_max_ankle',
# 'turn_duration_ankle',
# 'duration_rem_count',
# 'rem_percent',
# 'movement_rate']
free_living_features_highly_correlated = ['idle_minutes',
'turn_vel_std_ankle',
'swing',
'stance',
'duration_movement_count',
'turn_vel_max_ankle',
'turn_duration_ankle',
'duration_rem_count',
'rem_percent',
'movement_rate']
In [36]:
# 19 highly correlated at home features (structured activity)
# ['mean_pvt_delay_7_at_home',
# 'mobility_stance_at_home',
# 'mean_pvt_delay_at_home',
# 'pq_nondominant_rhythm_at_home',
# 'pq_nondominant_median_at_home',
# 'pq_dominant_rhythm_at_home',
# 'turn_vel_max_at_home',
# 'mobility_swing_at_home',
# 'zx_dominant_num_correct_at_home',
# 'turn_vel_std_at_home',
# 'turn_duration_ankle_at_home',
# 'turn_vel_max_ankle_at_home',
# 'mean_pvt_delay_5_at_home',
# 'zx_nondominant_median_at_home',
# 'zx_nondominant_num_correct_at_home',
# 'mean_pvt_delay_3_at_home',
# 'turn_vel_std_ankle_at_home',
# 'mobility_activity_at_home_time',
# 'mean_pvt_delay_1_at_home']
at_home_features_highly_correlated = ['mean_pvt_delay_7_at_home',
'mobility_stance_at_home',
'mean_pvt_delay_at_home',
'pq_nondominant_rhythm_at_home',
'pq_nondominant_median_at_home',
'pq_dominant_rhythm_at_home',
'turn_vel_max_at_home',
'mobility_swing_at_home',
'zx_dominant_num_correct_at_home',
'turn_vel_std_at_home',
'turn_duration_ankle_at_home',
'turn_vel_max_ankle_at_home',
'mean_pvt_delay_5_at_home',
'zx_nondominant_median_at_home',
'zx_nondominant_num_correct_at_home',
'mean_pvt_delay_3_at_home',
'turn_vel_std_ankle_at_home',
'mobility_activity_at_home_time',
'mean_pvt_delay_1_at_home']
In [203]:
def remove_outliers(feature_values, day_of_study):
# a function to remove outliers from input dataset and return filtered dataset as the ouput
m = 1.5 # deistance thereshold from the mean
mean = feature_values.mean()
std = feature_values.std()
tuples = list(zip(feature_values,day_of_study))
filtered_values = []
for (x,y) in tuples:
if (x >= mean - m * std) & (x <= mean + m * std):
filtered_values.append((x,y))
unzip_filtered_values = list(zip(*filtered_values))
# check for missing values
if len(unzip_filtered_values) > 0:
return pd.Series(list(unzip_filtered_values[0])), pd.Series(list(unzip_filtered_values[1]))
else:
return pd.Series([]),day_of_study
In [204]:
def standardize_axis(feature):
# a function to standardize the axis
# remove outliers (both feature values & associated days of study), return filtered values
# use the filtered values to assign a range to axis
# we assume dataframes FeatureDay and patient_IDs are already defined
All_filtered_feature_values = []
All_filtered_days_of_studies = []
# loop on all the patients
for ID in patient_IDs:
# extract part of FeatureDay that is related to a patient and input feature
col_1 = feature
col_2 = 'dayofstudy'
df = FeatureDay[FeatureDay['gls_subject_code'] == ID][[col_1,col_2]]
# sort the dataframe based on days of study
df.sort(col_2, inplace = True)
# create list of x: days of study, y: feature values
x = df[col_2]
y = df[col_1]
# remove outliers (both feature values & associated days of study)
y,x = remove_outliers(y,x)
# store all the filtered values
All_filtered_feature_values = All_filtered_feature_values + (pd.Series.tolist(y))
All_filtered_days_of_studies = All_filtered_days_of_studies + (pd.Series.tolist(x))
# set the axis ranges to the max value in the list of filtered values
max_y = (np.max(All_filtered_feature_values))
max_x = (np.max(All_filtered_days_of_studies))
# return ranges for axis
return max_y,max_x
In [205]:
def plot_feature_across_days(feature):
# a function to plot the measurments for a specific feature vs. days
figs, axes = plt.subplots(nrows= 5, ncols= 5,figsize=(20,20),dpi = 200)
print(feature)
# plot the measurments vs. days
for idx in range(len(patient_IDs)):
# extract the patient ID
ID = patient_IDs[idx]
# extract two columns as a dataframe
col_1 = feature
col_2 = 'dayofstudy'
df = FeatureDay[FeatureDay['gls_subject_code'] == ID][[col_1,col_2]]
# sort the dataframe based on days
df.sort(col_2, inplace = True)
x = df[col_2]
y = df[col_1]
# set the row and column numbers based on the fact that we have 25 patients
row = idx // 5
col = idx % 5
# standardize the axis
max_y,max_x = standardize_axis(feature)
axes[row,col].set_xlim(0, max_x)
axes[row,col].set_ylim(0, max_y)
axes[row,col].set_title(ID,y=0.9)
axes[row,col].set_xlabel('Days of Study')
axes[row,col].set_ylabel(feature)
# plot the measurments vs. days
if (len(y.unique()) == 1) & (np.isnan(y.unique()).sum() == 1):
pass
else:
y,x = remove_outliers(y,x)
if len(y) == 0:
pass
else:
sns.regplot(x,y,ax=axes[row,col])
axes[row,col].set_xlabel('Days of Study')
axes[row,col].set_ylabel(feature)
In [1]:
# distribution plot
# idle_minutes
# feature = free_living_features_highly_correlated[0]
# print(feature)
# figs, axes = plt.subplots(nrows= 5, ncols= 5,figsize=(20,20),dpi = 200)
# for idx in range(len(patient_IDs)):
# # extract the patient ID
# ID = patient_IDs[idx]
# # extract two columns as a dataframe
# col_1 = feature
# col_2 = 'dayofstudy'
# df = FeatureDay[FeatureDay['gls_subject_code'] == ID][[col_1,col_2]]
# # sort the dataframe based on days
# df.sort(col_2, inplace = True)
# # plot the measurments vs. days
# x = df[col_2]
# y = df[col_1].dropna()
# row = idx // 5
# col = idx % 5
# if (len(y.unique()) == 1) & (np.isnan(y.unique()).sum() == 1):
# axes[row,col].set_title(ID,y=0.9)
# else:
# sns.distplot(y,ax=axes[row,col])
# axes[row,col].set_title(ID,y=0.9)
In [25]:
# scatter plots of feature values
# try on the fetaure idle_minutes
feature = free_living_features_highly_correlated[0]
print(feature)
figs, axes = plt.subplots(nrows= 5, ncols= 5,figsize=(20,20),dpi = 200)
for idx in range(len(patient_IDs)):
# extract the patient ID
ID = patient_IDs[idx]
# extract two columns as a dataframe
col_1 = feature
col_2 = 'dayofstudy'
df = FeatureDay[FeatureDay['gls_subject_code'] == ID][[col_1,col_2]]
# sort the dataframe based on days
df.sort(col_2, inplace = True)
# plot the measurments vs. days
x = df[col_2]
y = df[col_1]
row = idx // 5
col = idx % 5
# standardize the exis
axes[row,col].set_xlim(0, FeatureDay[col_2].max())
axes[row,col].set_ylim(0, FeatureDay[col_1].max())
# plot feature values against days of study
axes[row,col].scatter(x,y)
axes[row,col].set_xlabel(col_2)
axes[row,col].set_ylabel(col_1)
axes[row,col].set_title(ID,y=0.9)
In [ ]:
# plot scatter plots of feature values against days of study
# plot the regression line that can best fit to the data
In [193]:
# idle_minutes
feature = free_living_features_highly_correlated[0]
plot_feature_across_days(feature)
In [194]:
# turn_vel_std_ankle
feature = free_living_features_highly_correlated[1]
plot_feature_across_days(feature)
In [195]:
# swing
feature = free_living_features_highly_correlated[2]
plot_feature_across_days(feature)
In [196]:
# stance
feature = free_living_features_highly_correlated[3]
plot_feature_across_days(feature)
In [197]:
# duration_movement_count
feature = free_living_features_highly_correlated[4]
plot_feature_across_days(feature)
In [198]:
# turn_vel_max_ankle
feature = free_living_features_highly_correlated[5]
plot_feature_across_days(feature)
In [199]:
# turn_duration_ankle
feature = free_living_features_highly_correlated[6]
plot_feature_across_days(feature)
In [200]:
# duration_rem_count
feature = free_living_features_highly_correlated[7]
plot_feature_across_days(feature)
In [201]:
# rem_percent
feature = free_living_features_highly_correlated[8]
plot_feature_across_days(feature)
In [202]:
# movement_rate
feature = free_living_features_highly_correlated[9]
plot_feature_across_days(feature)
In [174]:
# mean_pvt_delay_7_at_home
feature = at_home_features_highly_correlated[0]
plot_feature_across_days(feature)
In [175]:
# mobility_stance_at_home
feature = at_home_features_highly_correlated[1]
plot_feature_across_days(feature)
In [176]:
# mean_pvt_delay_at_home
feature = at_home_features_highly_correlated[2]
plot_feature_across_days(feature)
In [177]:
# pq_nondominant_rhythm_at_home
feature = at_home_features_highly_correlated[3]
plot_feature_across_days(feature)
In [178]:
# pq_nondominant_median_at_home
feature = at_home_features_highly_correlated[4]
plot_feature_across_days(feature)
In [179]:
# pq_dominant_rhythm_at_home
feature = at_home_features_highly_correlated[5]
plot_feature_across_days(feature)
In [180]:
# turn_vel_max_at_home
feature = at_home_features_highly_correlated[6]
plot_feature_across_days(feature)
In [181]:
# mobility_swing_at_home
feature = at_home_features_highly_correlated[7]
plot_feature_across_days(feature)
In [182]:
# zx_dominant_num_correct_at_home
feature = at_home_features_highly_correlated[8]
plot_feature_across_days(feature)
In [183]:
# turn_vel_std_at_home
feature = at_home_features_highly_correlated[9]
plot_feature_across_days(feature)
In [184]:
# turn_duration_ankle_at_home
feature = at_home_features_highly_correlated[10]
plot_feature_across_days(feature)
In [185]:
# turn_vel_max_ankle_at_home
feature = at_home_features_highly_correlated[11]
plot_feature_across_days(feature)
In [186]:
# mean_pvt_delay_5_at_home
feature = at_home_features_highly_correlated[12]
plot_feature_across_days(feature)
In [187]:
# zx_nondominant_median_at_home
feature = at_home_features_highly_correlated[13]
plot_feature_across_days(feature)
In [188]:
# zx_nondominant_num_correct_at_home
feature = at_home_features_highly_correlated[14]
plot_feature_across_days(feature)
In [189]:
# mean_pvt_delay_3_at_home
feature = at_home_features_highly_correlated[15]
plot_feature_across_days(feature)
In [190]:
# turn_vel_std_ankle_at_home
feature = at_home_features_highly_correlated[16]
plot_feature_across_days(feature)
In [191]:
# mobility_activity_at_home_time
feature = at_home_features_highly_correlated[17]
plot_feature_across_days(feature)
In [192]:
# mean_pvt_delay_1_at_home
feature = at_home_features_highly_correlated[18]
plot_feature_across_days(feature)
In [ ]: